package org.hong.monkey.crawl

import org.apache.http.HttpHost
import org.apache.http.HttpResponse
import org.apache.http.HttpStatus
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.params.ClientPNames
import org.apache.http.conn.params.ConnRoutePNames
import org.apache.http.cookie.params.CookieSpecPNames
import org.apache.http.cookie.{CookieOrigin, Cookie, CookieSpec, CookieSpecFactory}
import org.apache.http.impl.client.{BasicCookieStore, DefaultHttpClient}
import org.apache.http.impl.cookie.BrowserCompatSpec
import org.apache.http.params.{HttpParams, CoreConnectionPNames}
import org.apache.http.util.EntityUtils
import org.hong.monkey.Logging

import scala.concurrent.duration._

import scala.concurrent.{ExecutionContext, Await, Future}

/**
 * Created by open on 2016/12/12.
 */

private[monkey]
class HttpClientDownloader extends HttpDownloader with Logging {

  override def get (crawlRequest: CrawlRequest)(implicit executor: ExecutionContext) : CrawlResponse = {
    val httpClient = new DefaultHttpClient()
    val manager = httpClient.getConnectionManager()
    val request = new HttpGet(crawlRequest.link)

    val timeout = 60000
    httpClient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout);
    httpClient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT, timeout);

    // 解决 WARN: ResponseProcessCookies
    val cookieStore = new BasicCookieStore();
    //Bind custom cookie store to the local context
    httpClient.setCookieStore(cookieStore);
    val csf = new CookieSpecFactory() {
      def newInstance(params:HttpParams):CookieSpec = {
        return new BrowserCompatSpec() {
          override def validate(cookie:Cookie, origin:CookieOrigin) {}
        }
      }
    }

    httpClient.getCookieSpecs().register("monkey", csf);
    httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY, "monkey");
    httpClient.getParams().setParameter(CookieSpecPNames.SINGLE_COOKIE_HEADER, true);

    if (crawlRequest.cookies != null && crawlRequest.cookies != "") {
      request.setHeader("Cookie", crawlRequest.cookies)
    }

    if (crawlRequest.headers != null) {
      val keys = crawlRequest.headers.keySet().iterator
      while (keys.hasNext) {
        val key = keys.next()
        val value = crawlRequest.headers.get(key)
        request.setHeader(key, value)
      }
    }

    if (crawlRequest.proxy != null) {
      httpClient.getCredentialsProvider().setCredentials(
        new AuthScope(crawlRequest.proxy.host, crawlRequest.proxy.port),
        new UsernamePasswordCredentials(crawlRequest.proxy.username, crawlRequest.proxy.password))

      val httpProxy = new HttpHost(crawlRequest.proxy.host, crawlRequest.proxy.port)
      httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, httpProxy)
    }

    var response:HttpResponse = null
    try {
      response = httpClient.execute(request)
      if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
        return new CrawlResponse(crawlRequest.link, "", null, response.getStatusLine().getStatusCode(),
          new MonkeyDownloaderException(new Exception("HTTP code : " + response.getStatusLine().getStatusCode())),
          null, crawlRequest.proxy, crawlRequest.carryData)
      }
    } catch {
      case e:Exception => {
        var responseCode = -10;
        if (response != null) {
          responseCode = response.getStatusLine().getStatusCode()
          request.abort()
          request.releaseConnection()
        }

        return new CrawlResponse(crawlRequest.link, "", null, responseCode,
          new MonkeyDownloaderException(e), null, crawlRequest.proxy, crawlRequest.carryData)
      }
    }

    val future = Future {

      try {
        if (response.getEntity() != null) {
          val bytes = EntityUtils.toByteArray(response.getEntity())

          var rheaders = new Headers
          val hs = response.getAllHeaders()
          for (h <- hs)
            rheaders.put(h.getName(), h.getValue())

          val rcookies = httpClient.getCookieStore().getCookies()
          var cookiesList = List.empty[String]
          for (i <- 0 until rcookies.size()) {
            val cookieString = rcookies.get(i).getName()+"="+rcookies.get(i).getValue()
            cookiesList = cookieString +: cookiesList
          }

          new CrawlResponse(crawlRequest.link, cookiesList.mkString(";"), rheaders, response.getStatusLine().getStatusCode(),
            null, bytes, crawlRequest.proxy, crawlRequest.carryData)
        } else
          new CrawlResponse(crawlRequest.link, "", null, -20,
            new MonkeyDownloaderException(new Exception("Response entity is null")),
            null, crawlRequest.proxy, crawlRequest.carryData)
      } catch {
        case e:Exception =>
          new CrawlResponse(crawlRequest.link, "", null, -30,
            new MonkeyDownloaderException(e), null, crawlRequest.proxy, crawlRequest.carryData)
      }
    }

    val content = {
      try {
        val wait = if (timeout < 1000) 3 else (timeout / 1000) * 2
        Await.result(future, wait.second)
      } catch {
        case e:Exception => {
          request.abort()
          request.releaseConnection()
          new CrawlResponse(crawlRequest.link, "", null, -40,
            new MonkeyDownloaderException(e), null, crawlRequest.proxy, crawlRequest.carryData)
        }
      }
    }

    manager.shutdown()

    content
  }
}
